
from nbconvert import HTMLExporter
from nbformat import read
from nbformat import NO_CONVERT

with open('your_notebook.ipynb') as f:
    notebook = read(f, as_version=NO_CONVERT)

html_exporter = HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(notebook)

with open('output.html', 'w') as f:
    f.write(body)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[1], line 5
      2 from nbformat import read
      3 from nbformat import NO_CONVERT
----> 5 with open('your_notebook.ipynb') as f:
      6     notebook = read(f, as_version=NO_CONVERT)
      8 html_exporter = HTMLExporter()

File ~\anaconda3\Lib\site-packages\IPython\core\interactiveshell.py:286, in _modified_open(file, *args, **kwargs)
    279 if file in {0, 1, 2}:
    280     raise ValueError(
    281         f"IPython won't let you open fd={file} by default "
    282         "as it is likely to crash IPython. If you know what you are doing, "
    283         "you can use builtins' open."
    284     )
--> 286 return io_open(file, *args, **kwargs)

FileNotFoundError: [Errno 2] No such file or directory: 'your_notebook.ipynb'


!pip install pydicom

Collecting pydicom
  Downloading pydicom-3.0.1-py3-none-any.whl.metadata (9.4 kB)
Downloading pydicom-3.0.1-py3-none-any.whl (2.4 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.4/2.4 MB 66.0 MB/s eta 0:00:00
Installing collected packages: pydicom
Successfully installed pydicom-3.0.1


!pip install scikit-multilearn

Collecting scikit-multilearn
  Downloading scikit_multilearn-0.2.0-py3-none-any.whl.metadata (6.0 kB)
Downloading scikit_multilearn-0.2.0-py3-none-any.whl (89 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 89.4/89.4 kB 8.3 MB/s eta 0:00:00
Installing collected packages: scikit-multilearn
Successfully installed scikit-multilearn-0.2.0


!pip install grad-cam

Collecting grad-cam
  Downloading grad-cam-1.5.4.tar.gz (7.8 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.8/7.8 MB 85.1 MB/s eta 0:00:00
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
  Preparing metadata (pyproject.toml) ... done
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from grad-cam) (1.26.4)
Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from grad-cam) (11.0.0)
Requirement already satisfied: torch>=1.7.1 in /usr/local/lib/python3.10/dist-packages (from grad-cam) (2.5.1+cu121)
Requirement already satisfied: torchvision>=0.8.2 in /usr/local/lib/python3.10/dist-packages (from grad-cam) (0.20.1+cu121)
Collecting ttach (from grad-cam)
  Downloading ttach-0.0.3-py3-none-any.whl.metadata (5.2 kB)
Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from grad-cam) (4.67.1)
Requirement already satisfied: opencv-python in /usr/local/lib/python3.10/dist-packages (from grad-cam) (4.10.0.84)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from grad-cam) (3.8.0)
Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from grad-cam) (1.6.0)
Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (3.16.1)
Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (4.12.2)
Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (3.4.2)
Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (3.1.4)
Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (2024.10.0)
Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.7.1->grad-cam) (1.13.1)
Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch>=1.7.1->grad-cam) (1.3.0)
Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (1.3.1)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (4.55.3)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (1.4.7)
Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (24.2)
Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (3.2.0)
Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->grad-cam) (2.8.2)
Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->grad-cam) (1.13.1)
Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->grad-cam) (1.4.2)
Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->grad-cam) (3.5.0)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->grad-cam) (1.17.0)
Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.7.1->grad-cam) (3.0.2)
Downloading ttach-0.0.3-py3-none-any.whl (9.8 kB)
Building wheels for collected packages: grad-cam
  Building wheel for grad-cam (pyproject.toml) ... done
  Created wheel for grad-cam: filename=grad_cam-1.5.4-py3-none-any.whl size=39588 sha256=8ac4d2fd2a1eb1d54b573a21b531b30fb9c7332aedea84fd55a8ead7f0258aea
  Stored in directory: /root/.cache/pip/wheels/50/b0/82/1f97b5348c7fe9f0ce0ba18497202cafa5dec4562bd5292680
Successfully built grad-cam
Installing collected packages: ttach, grad-cam
Successfully installed grad-cam-1.5.4 ttach-0.0.3


!pip install opencv-python

Requirement already satisfied: opencv-python in /usr/local/lib/python3.10/dist-packages (4.10.0.84)
Requirement already satisfied: numpy>=1.21.2 in /usr/local/lib/python3.10/dist-packages (from opencv-python) (1.26.4)


# import libraries

# basic
import warnings
warnings.filterwarnings('ignore')

import os
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'
import random
import pydicom
import itertools
import cv2
import numpy as np
import pandas as pd
import math
from tqdm.notebook import tqdm
from sklearn.preprocessing import MultiLabelBinarizer
from skmultilearn.model_selection import iterative_train_test_split
# visualization
from PIL import Image
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import matplotlib.gridspec as gridspec
# PyTorch
import torch
import torchvision
from torch.utils.data import Dataset, DataLoader
from torchvision import models
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.rpn import AnchorGenerator
from torchvision.models.detection import FasterRCNN
from torchvision.transforms import v2
from torchvision import tv_tensors
from torchvision.tv_tensors import BoundingBoxes

# object detection
!pip install pycocotools
import pycocotools
from pycocotools.coco import COCO

# object detection
import json
from skimage.measure import label as sk_label
from skimage.measure import regionprops as sk_regions

import pytorch_grad_cam
from pytorch_grad_cam import EigenCAM, AblationCAM
from pytorch_grad_cam.ablation_layer import AblationLayerFasterRCNN
from pytorch_grad_cam.utils.model_targets import FasterRCNNBoxScoreTarget
from pytorch_grad_cam.utils.reshape_transforms import fasterrcnn_reshape_transform
from pytorch_grad_cam.utils.image import show_cam_on_image

Requirement already satisfied: pycocotools in /usr/local/lib/python3.10/dist-packages (2.0.8)
Requirement already satisfied: matplotlib>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from pycocotools) (3.8.0)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from pycocotools) (1.26.4)
Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (1.3.1)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (4.55.3)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (1.4.7)
Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (24.2)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (11.0.0)
Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (3.2.0)
Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools) (2.8.2)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=2.1.0->pycocotools) (1.17.0)


!wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/engine.py
!wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/utils.py
!wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_utils.py
!wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_eval.py
!wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/transforms.py

from engine import evaluate

--2024-12-30 05:02:19--  https://raw.githubusercontent.com/pytorch/vision/main/references/detection/engine.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 4063 (4.0K) [text/plain]
Saving to: ‘engine.py’

engine.py           100%[===================>]   3.97K  --.-KB/s    in 0s      

2024-12-30 05:02:19 (83.7 MB/s) - ‘engine.py’ saved [4063/4063]

--2024-12-30 05:02:19--  https://raw.githubusercontent.com/pytorch/vision/main/references/detection/utils.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 8388 (8.2K) [text/plain]
Saving to: ‘utils.py’

utils.py            100%[===================>]   8.19K  --.-KB/s    in 0s      

2024-12-30 05:02:19 (97.3 MB/s) - ‘utils.py’ saved [8388/8388]

--2024-12-30 05:02:19--  https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_utils.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 8397 (8.2K) [text/plain]
Saving to: ‘coco_utils.py’

coco_utils.py       100%[===================>]   8.20K  --.-KB/s    in 0s      

2024-12-30 05:02:20 (86.4 MB/s) - ‘coco_utils.py’ saved [8397/8397]

--2024-12-30 05:02:20--  https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_eval.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 6447 (6.3K) [text/plain]
Saving to: ‘coco_eval.py’

coco_eval.py        100%[===================>]   6.30K  --.-KB/s    in 0s      

2024-12-30 05:02:20 (86.6 MB/s) - ‘coco_eval.py’ saved [6447/6447]

--2024-12-30 05:02:20--  https://raw.githubusercontent.com/pytorch/vision/main/references/detection/transforms.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.110.133, 185.199.109.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 23628 (23K) [text/plain]
Saving to: ‘transforms.py’

transforms.py       100%[===================>]  23.07K  --.-KB/s    in 0s      

2024-12-30 05:02:21 (141 MB/s) - ‘transforms.py’ saved [23628/23628]


from google.colab import drive
drive.mount('/content/drive')

data_path = '/content/drive/MyDrive/processed_images'
print("目錄內容：", os.listdir(data_path))  # 檢查檔案是否存在

Mounted at /content/drive
目錄內容： ['train.csv', 'test.csv', 'train', 'test']


## TODO: Prepare your own information
class config:

    ## roots for training & valid
    root = "/content/drive/MyDrive/processed_images/train"
    info_root = "/content/drive/MyDrive/processed_images/train"
    save_root = "/content/drive/MyDrive/working/"
    #root = "/kaggle/input/dlmia-hw05-new"
    #info_root = "/kaggle/input/dlmia-hw05-new"
    #save_root = "/kaggle/working/"

    ## for test images
    test_root = "/content/drive/MyDrive/processed_images/test"
    info_root_test = "/content/drive/MyDrive/processed_images/test"
    #test_root = '/kaggle/input/dlmia-hw05-test'
    #info_root_test = '/kaggle/input/dlmia-hw05-test'

    num_classes = 8 #(for fasterrcnn: background + # of classes): 1+7=8

    batch_size = 4
    epochs = 20
    weight_decay = 1e-7
    lr = 1e-3
    momentum = 0.9 ###SGD
    seed = 42
    workers = 4
    categories = ['normal', 'aortic_curvature', 'aortic_atherosclerosis_calcification',
                  'cardiac_hypertrophy', 'intercostal_pleural_thickening', 'lung_field_infiltration',
                  'degenerative_joint_disease_of_the_thoracic_spine', 'scoliosis']
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")


def seed_everything(seed):

    random.seed(seed) # Set Python random seed
    np.random.seed(seed) # Set NumPy random seed
    torch.manual_seed(seed) # Set PyTorch random seed for CPU and GPU
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)

    # Set PyTorch deterministic operations for cudnn backend
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False


seed_everything(config.seed)


annfile = config.info_root + "/train.json"
#annfile = '/content/drive/MyDrive/train.json'
coco = COCO(annfile)   ###object(dataloader比較方便)
coco.cats

loading annotations into memory...
Done (t=0.00s)
creating index...
index created!

{1: {'id': 1, 'name': 'aortic_curvature'},
 2: {'id': 2, 'name': 'aortic_atherosclerosis_calcification'},
 3: {'id': 3, 'name': 'cardiac_hypertrophy'},
 4: {'id': 4, 'name': 'intercostal_pleural_thickening'},
 5: {'id': 5, 'name': 'lung_field_infiltration'},
 6: {'id': 6, 'name': 'degenerative_joint_disease_of_the_thoracic_spine'},
 7: {'id': 7, 'name': 'scoliosis'}}


coco.loadImgs(0)

[{'file_name': 'normal/220_94.jpg', 'height': 2560, 'width': 2472, 'id': 0}]


ann_ids = coco.getAnnIds(imgIds = 0)
coco.loadAnns(ann_ids)

[]


###目前id較原本的sample多一
ann_ids = coco.getAnnIds(imgIds = 155)
coco.loadAnns(ann_ids)

[{'id': 165,
  'image_id': 155,
  'category_id': 3,
  'bbox': [816, 936, 1048, 896],
  'area': 939008,
  'iscrowd': 0,
  'segmentation': [[816, 936, 1864, 936, 1864, 1832, 816, 1832]]},
 {'id': 166,
  'image_id': 155,
  'category_id': 5,
  'bbox': [144, 336, 2008, 1941],
  'area': 3897528,
  'iscrowd': 0,
  'segmentation': [[144, 336, 2152, 336, 2152, 2277, 144, 2277]]}]


del coco


## TO DO: Augmentation?No
class medTransform:
    def __init__(self, train=False):
        if train:
            self.transforms = v2.Compose(
                [
                    v2.ToImage(), ## Used while using PIL image
                    v2.ToDtype(torch.float32, scale=True),
                ]
            )
        else:
            self.transforms = v2.Compose(
                [
                    v2.ToImage(), ## Used while using PIL image
                    v2.ToDtype(torch.float32, scale=True),
                ]
            )

    def __call__(self, x, bboxes):
        if isinstance(x, torch.Tensor):
            height, width = x.shape[-2], x.shape[-1]  # (C, H, W) format
        else:  # Assuming x is a PIL Image
            width, height = x.size
        ## Loading format is COCO bboxes[x,y,w,h](吃左上XY和右下的XY)
        bboxes = tv_tensors.BoundingBoxes(bboxes, format="XYWH", canvas_size=(height,width))
        return self.transforms(x, bboxes)


class MedDataset(Dataset):

    def __init__(self, root, info_root, split, transforms = None):
        self.split = split
        self.root = root
        self.info_root = info_root
        self.transforms = transforms
        self.coco = COCO(os.path.join(self.info_root, f"{self.split}.json"))
        self.ids = list(sorted(self.coco.imgs.keys()))

    def get_image(self, img_id: int):
        image_path = os.path.join(self.root,self.coco.loadImgs(img_id)[0]['file_name'])
        image = Image.open(image_path).convert("RGB")
        return image

    def get_annotation(self, img_id: int):
        return self.coco.loadAnns(self.coco.getAnnIds(img_id))

    def __getitem__(self, index):
        normal = False
        img_id = self.ids[index]
        image = self.get_image(img_id)
        annotation = self.get_annotation(img_id)

        bboxes = [a['bbox']  for a in annotation]
        category_ids = [a['category_id']  for a in annotation]
        if bboxes == []:
            normal = True

        if self.transforms:
            image, bboxes = self.transforms(image, bboxes)

        def reformat_bboxes(boxes):
            return [[val[0], val[1], val[0] + val[2], val[1] + val[3]] for val in boxes]

        if normal != True:
            ## Recall that the original format is COCO
            bboxes = reformat_bboxes(bboxes)

        def create_target(bboxes, normal):
            if normal:
                return {
                    'boxes': torch.zeros((0, 4), dtype=torch.float32),  # Empty boxes
                    'labels': torch.tensor(category_ids, dtype=torch.int64),
                    'image_id': img_id,
                    'area': torch.zeros((0,), dtype=torch.float32),  # Empty areas
                    'iscrowd': torch.zeros((0,), dtype=torch.int64),  # Empty tensor for iscrowd
                }
            else:

                return {
                    'boxes': torch.tensor(bboxes),
                    'labels': torch.tensor(category_ids, dtype=torch.int64),
                    'image_id': img_id,
                    'area': torch.tensor([(bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) for bbox in bboxes], dtype=torch.float32),
                    'iscrowd': torch.tensor([a['iscrowd'] for a in annotation], dtype=torch.int64)
                }

        targets = create_target(bboxes,normal)
        return image, targets

    def __len__(self):
        return len(self.ids)


def collate_fn(batch: list[torch.tensor, dict]):
    return tuple(zip(*batch))


def plot_image_with_boxes(image_tensor, boxes_dict):
    image_np = image_tensor.permute(1, 2, 0).numpy()
    fig, ax = plt.subplots(1)
    # Display the image
    ax.imshow(image_np)
    for box in boxes_dict['boxes']:
        # Extract coordinates (x0, y0, x1, y1)
        x0, y0, x1, y1 = box
        # Calculate the height as (y0 - y1) since y0 is the top and y1 is the bottom
        height = y1 - y0
        # Create a rectangle patch with (x0, y0) as the top-left corner
        rect = patches.Rectangle((x0, y0), x1 - x0, height, linewidth=2, edgecolor='r', facecolor='none')
        ax.add_patch(rect)
    plt.show()


train_dataset = MedDataset(root = config.root, info_root = config.info_root, split = "train", transforms = medTransform(train=True))
train_loader = DataLoader(train_dataset, batch_size = config.batch_size, shuffle = True,num_workers=4, collate_fn = collate_fn)

loading annotations into memory...
Done (t=0.00s)
creating index...
index created!


a,b = train_dataset.__getitem__(1)
plot_image_with_boxes(a,b)


a,b = train_dataset.__getitem__(155)
print(b)
plot_image_with_boxes(a,b)

{'boxes': tensor([[ 816,  936, 1864, 1832],
        [ 144,  336, 2152, 2277]]), 'labels': tensor([3, 5]), 'image_id': 155, 'area': tensor([ 939008., 3897528.]), 'iscrowd': tensor([0, 0])}


def fasterrcnn(num_classes):
    model = models.detection.fasterrcnn_resnet50_fpn(weights='COCO_V1')
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    model.roi_heads.box_predictor = None
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

    return model


model = fasterrcnn(config.num_classes)


def train_one_epoch(model, train_loader, optimizer, epoch, device):
    model.train()

    train_loss = []
    train_loss_dict = []

    lr_scheduler = None

    for images, targets in tqdm(train_loader):
        images = [image.to(device) for image in images]
        targets = [{k: (torch.tensor(v,device=device) if not isinstance(v, torch.Tensor) else v.to(device)) for k, v in t.items()} for t in targets]

        loss_dict = model(images, targets)
        losses = sum(loss for loss in loss_dict.values())

        batch_loss_value = losses.item()
        batch_loss_dict = {k: v.item() for k, v in loss_dict.items()}

        train_loss.append(batch_loss_value)
        train_loss_dict.append(batch_loss_dict)

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()

    train_loss = np.mean(train_loss)
    train_loss_dict = pd.DataFrame(train_loss_dict).mean()
    train_loss_classifier = train_loss_dict.loss_classifier
    train_loss_box_reg = train_loss_dict.loss_box_reg
    train_loss_rpn_box_reg = train_loss_dict.loss_rpn_box_reg
    train_loss_objectness = train_loss_dict.loss_objectness

    return train_loss, train_loss_classifier, train_loss_box_reg, train_loss_rpn_box_reg, train_loss_objectness


def validation(model, val_loader, device):
    model.train()
    #model.eval()
    for m in model.modules():
        if isinstance(m, torchvision.ops.Conv2dNormActivation):
            m.eval()
        if isinstance(m, torchvision.ops.FrozenBatchNorm2d):
            m.eval()
        if isinstance(m, torch.nn.BatchNorm2d):
            m.eval()
    val_loss = []
    val_loss_dict = []
    with torch.no_grad():
        for images, targets in tqdm(val_loader):
            images = [image.to(device) for image in images]
            targets = [{k: (torch.tensor(v,device=device) if not isinstance(v, torch.Tensor) else v.to(device)) for k, v in t.items()} for t in targets]

            loss = model(images, targets)
            total_loss = sum(l for l in loss.values())

            loss_value = total_loss.item()
            loss_dict = {k: v.item() for k, v in loss.items()}

            val_loss.append(loss_value)
            val_loss_dict.append(loss_dict)

    val_loss = np.mean(val_loss)

    val_loss_dict = pd.DataFrame(val_loss_dict).mean()
    val_loss_classifier = val_loss_dict.loss_classifier
    val_loss_box_reg = val_loss_dict.loss_box_reg
    val_loss_rpn_box_reg = val_loss_dict.loss_rpn_box_reg
    val_loss_objectness = val_loss_dict.loss_objectness

    return val_loss, val_loss_classifier, val_loss_box_reg, val_loss_rpn_box_reg, val_loss_objectness


del model
del train_dataset, train_loader


def seed_worker(worker_id):
    worker_seed = torch.initial_seed() % 2**32
    np.random.seed(worker_seed)
    random.seed(worker_seed)


def train_one_epoch(model, train_loader, optimizer, epoch, device):
    model.train()

    train_loss = []
    train_loss_dict = []

    lr_scheduler = None

    for images, targets in tqdm(train_loader):
        images = [image.to(device) for image in images]
        targets = [{k: (torch.tensor(v,device=device) if not isinstance(v, torch.Tensor) else v.to(device)) for k, v in t.items()} for t in targets]

        loss_dict = model(images, targets)
        losses = sum(loss for loss in loss_dict.values())

        batch_loss_value = losses.item()
        batch_loss_dict = {k: v.item() for k, v in loss_dict.items()}

        train_loss.append(batch_loss_value)
        train_loss_dict.append(batch_loss_dict)

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()

    train_loss = np.mean(train_loss)
    train_loss_dict = pd.DataFrame(train_loss_dict).mean()
    train_loss_classifier = train_loss_dict.loss_classifier
    train_loss_box_reg = train_loss_dict.loss_box_reg
    train_loss_rpn_box_reg = train_loss_dict.loss_rpn_box_reg
    train_loss_objectness = train_loss_dict.loss_objectness

    return train_loss, train_loss_classifier, train_loss_box_reg, train_loss_rpn_box_reg, train_loss_objectness


def main():

    seed_everything(config.seed)
    g = torch.Generator()
    g.manual_seed(config.seed)

    train_dataset = MedDataset(root = config.root, info_root = config.info_root, split = "train", transforms = medTransform(train=True))
    val_dataset = MedDataset(root = config.root, info_root = config.info_root, split = "val",  transforms = medTransform(train=False))

    train_loader = DataLoader(train_dataset, batch_size = config.batch_size, shuffle = True,
                              num_workers=config.workers, collate_fn = collate_fn,pin_memory=True
                             )
    val_loader = DataLoader(val_dataset, batch_size = config.batch_size, shuffle = False,
                            num_workers=config.workers, worker_init_fn=seed_worker,
                            generator=g, collate_fn = collate_fn,pin_memory=True
                           )


    device = config.device
    model =  fasterrcnn(num_classes = config.num_classes)
    model.to(device)

    parameters = [p for p in model.parameters() if p.requires_grad]

    optimizer = torch.optim.SGD(parameters, lr = config.lr, momentum = config.momentum, nesterov = True, weight_decay = config.weight_decay)

    #scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs, eta_min=0)

    best_val_loss = float("inf")
    best_map50 = 0.0
    history = {
        "train": {
            "loss": [],
            "loss_classifier": [],
            "loss_box_reg": [],
            "loss_rpn_box_reg": [],
            "loss_objectness": []
        },
        "val": {
            "loss": [],
            "loss_classifier": [],
            "loss_box_reg": [],
            "loss_rpn_box_reg": [],
            "loss_objectness": []
        },
        "map50":{
            "train":[],
            "valid":[],
        }
    }
    best_idx = 0
    print('start')
    for epoch in range(config.epochs):
        print()
        train_loss, train_loss_classifier, train_loss_box_reg, train_loss_rpn_box_reg, train_loss_objectness = train_one_epoch(
            model, train_loader, optimizer, epoch, device,
        )

        val_loss, val_loss_classifier, val_loss_box_reg, val_loss_rpn_box_reg, val_loss_objectness = validation(
            model, val_loader, device
        )

        ## Training
        history["train"]["loss"].append(train_loss)
        history["train"]["loss_classifier"].append(train_loss_classifier)
        history["train"]["loss_box_reg"].append(train_loss_box_reg)
        history["train"]["loss_rpn_box_reg"].append(train_loss_rpn_box_reg)
        history["train"]["loss_objectness"].append(train_loss_objectness)
        ## Validation
        history["val"]["loss"].append(val_loss)
        history["val"]["loss_classifier"].append(val_loss_classifier)
        history["val"]["loss_box_reg"].append(val_loss_box_reg)
        history["val"]["loss_rpn_box_reg"].append(val_loss_rpn_box_reg)
        history["val"]["loss_objectness"].append(val_loss_objectness)


        print(f'Epoch: {epoch+1}/{config.epochs} | LR: {optimizer.state_dict()["param_groups"][0]["lr"]:.6f}')

        print("*****Training*****")
        print(f'Loss: {train_loss:.4f} | Classifier Loss: {train_loss_classifier:.4f} | Box Reg Loss: {train_loss_box_reg:.4f} | RPN Box Reg Loss: {train_loss_rpn_box_reg:.4f} | Objectness Loss: {train_loss_objectness:.4f}')
        train_evaluator = evaluate(model, train_loader, device = device)
        print("*****Validation*****")
        print(f'Loss: {val_loss:.4f} | Classifier Loss: {val_loss_classifier:.4f} | Box Reg Loss: {val_loss_box_reg:.4f} | RPN Box Reg Loss: {val_loss_rpn_box_reg:.4f} | Objectness Loss: {val_loss_objectness:.4f}')
        valid_evaluator = evaluate(model, val_loader, device = device)

        train_map50 = train_evaluator.coco_eval['bbox'].stats[1]
        valid_map50 = valid_evaluator.coco_eval['bbox'].stats[1]
        print("*****Training*****")
        print(f'MAP: {train_map50:.4f}')
        print("*****Validationg*****")
        print(f'MAP: {valid_map50:.4f}')
        history["map50"]["train"].append(train_map50)
        history["map50"]["valid"].append(valid_map50)

        ## TODO save your model

        if valid_map50 > best_map50:
            save_file = {
                "model": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "epoch": epoch,
                "args": config
            }
            best_idx=epoch
            best_map50 = valid_map50
            torch.save(save_file, os.path.join(config.save_root,"final.pth"))
        #scheduler.step()
    print(f'Best epoch in {best_idx+1}')


    ## Evaluation result
    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["map50"]["train"], label = 'Training map50')
    plt.plot(range(config.epochs), history["map50"]["valid"], label = 'Validation map50')
    plt.xlabel('Epoch')
    plt.ylabel('map')
    plt.legend()
    plt.title('Training and Validation map50')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss"], label = 'Training Loss')
    plt.plot(range(config.epochs), history["val"]["loss"], label = 'Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title('Training and Validation Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_classifier"], label = 'Training Classifier Loss')
    plt.plot(range(config.epochs), history["val"]["loss_classifier"], label = 'Validation Classifier Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Classifier Loss')
    plt.legend()
    plt.title('Training and Validation Classifier Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_box_reg"], label = 'Training Box Reg Loss')
    plt.plot(range(config.epochs), history["val"]["loss_box_reg"], label = 'Validation Box Reg Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Box Reg Loss')
    plt.legend()
    plt.title('Training and Validation Box Reg Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_rpn_box_reg"], label = 'Training RPN Box Reg Loss')
    plt.plot(range(config.epochs), history["val"]["loss_rpn_box_reg"], label = 'Validation RPN Box Reg Loss')

    plt.xlabel('Epoch')
    plt.ylabel('RPN Box Reg Loss')
    plt.legend()
    plt.title('Training and Validation RPN Box Reg Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_objectness"], label = 'Training Objectness Loss')
    plt.plot(range(config.epochs), history["val"]["loss_objectness"], label = 'Validation Objectness Loss')

    plt.xlabel('Epoch')
    plt.ylabel('Objectness Loss')
    plt.legend()
    plt.title('Training and Validation Objectness Loss Curves')
    plt.show()


## IMAGENET 3
if __name__ == "__main__":
    main()

loading annotations into memory...
Done (t=0.00s)
creating index...
index created!
loading annotations into memory...
Done (t=0.00s)
creating index...
index created!
start

  0%|          | 0/35 [00:00<?, ?it/s]

  0%|          | 0/10 [00:00<?, ?it/s]

Epoch: 1/20 | LR: 0.001000
*****Training*****
Loss: 0.4064 | Classifier Loss: 0.2827 | Box Reg Loss: 0.0945 | RPN Box Reg Loss: 0.0096 | Objectness Loss: 0.0197
creating index...
index created!
Test:  [ 0/35]  eta: 0:01:38  model_time: 0.4251 (0.4251)  evaluator_time: 0.0251 (0.0251)  time: 2.8075  data: 2.3102  max mem: 9591
Test:  [34/35]  eta: 0:00:00  model_time: 0.4136 (0.4146)  evaluator_time: 0.0108 (0.0123)  time: 0.4644  data: 0.0004  max mem: 9591
Test: Total time: 0:00:18 (0.5384 s / it)
Averaged stats: model_time: 0.4136 (0.4146)  evaluator_time: 0.0108 (0.0123)
Accumulating evaluation results...
DONE (t=0.14s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.007
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.033
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.007
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.053
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.129
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.129
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.129
*****Validation*****
Loss: 0.2334 | Classifier Loss: 0.1220 | Box Reg Loss: 0.0929 | RPN Box Reg Loss: 0.0075 | Objectness Loss: 0.0110
creating index...
index created!
Test:  [ 0/10]  eta: 0:00:27  model_time: 0.4217 (0.4217)  evaluator_time: 0.0075 (0.0075)  time: 2.7007  data: 2.2229  max mem: 9591
Test:  [ 9/10]  eta: 0:00:00  model_time: 0.4193 (0.3821)  evaluator_time: 0.0091 (0.0112)  time: 0.6554  data: 0.2227  max mem: 9591
Test: Total time: 0:00:06 (0.6634 s / it)
Averaged stats: model_time: 0.4193 (0.3821)  evaluator_time: 0.0091 (0.0112)
Accumulating evaluation results...
DONE (t=0.05s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.010
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.039
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.001
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.010
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.073
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.152
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.152
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.152
*****Training*****
MAP: 0.0334
*****Validationg*****
MAP: 0.0392

  0%|          | 0/35 [00:00<?, ?it/s]

  0%|          | 0/10 [00:00<?, ?it/s]

Epoch: 2/20 | LR: 0.001000
*****Training*****
Loss: 0.2366 | Classifier Loss: 0.1184 | Box Reg Loss: 0.0998 | RPN Box Reg Loss: 0.0087 | Objectness Loss: 0.0097
creating index...
index created!
Test:  [ 0/35]  eta: 0:01:31  model_time: 0.4098 (0.4098)  evaluator_time: 0.0207 (0.0207)  time: 2.6037  data: 2.1260  max mem: 9591
Test:  [34/35]  eta: 0:00:00  model_time: 0.4070 (0.4137)  evaluator_time: 0.0152 (0.0153)  time: 0.4597  data: 0.0004  max mem: 9591
Test: Total time: 0:00:18 (0.5353 s / it)
Averaged stats: model_time: 0.4070 (0.4137)  evaluator_time: 0.0152 (0.0153)
Accumulating evaluation results...
DONE (t=0.20s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.016
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.064
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.001
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.016
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.078
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.241
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.243
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.243
*****Validation*****
Loss: 0.2171 | Classifier Loss: 0.1119 | Box Reg Loss: 0.0906 | RPN Box Reg Loss: 0.0067 | Objectness Loss: 0.0079
creating index...
index created!
Test:  [ 0/10]  eta: 0:00:27  model_time: 0.4131 (0.4131)  evaluator_time: 0.0169 (0.0169)  time: 2.7162  data: 2.2377  max mem: 9591
Test:  [ 9/10]  eta: 0:00:00  model_time: 0.4131 (0.3838)  evaluator_time: 0.0169 (0.0178)  time: 0.6653  data: 0.2242  max mem: 9591
Test: Total time: 0:00:06 (0.6730 s / it)
Averaged stats: model_time: 0.4131 (0.3838)  evaluator_time: 0.0169 (0.0178)
Accumulating evaluation results...
DONE (t=0.07s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.027
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.087
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.007
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.027
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.117
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.260
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.269
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.269
*****Training*****
MAP: 0.0639
*****Validationg*****
MAP: 0.0869

  0%|          | 0/35 [00:00<?, ?it/s]

  0%|          | 0/10 [00:00<?, ?it/s]

Epoch: 3/20 | LR: 0.001000
*****Training*****
Loss: 0.2256 | Classifier Loss: 0.1110 | Box Reg Loss: 0.0983 | RPN Box Reg Loss: 0.0087 | Objectness Loss: 0.0075
creating index...
index created!
Test:  [ 0/35]  eta: 0:01:31  model_time: 0.4049 (0.4049)  evaluator_time: 0.0253 (0.0253)  time: 2.6173  data: 2.1406  max mem: 9591
Test:  [34/35]  eta: 0:00:00  model_time: 0.4197 (0.4145)  evaluator_time: 0.0139 (0.0229)  time: 0.4695  data: 0.0004  max mem: 9591
Test: Total time: 0:00:19 (0.5440 s / it)
Averaged stats: model_time: 0.4197 (0.4145)  evaluator_time: 0.0139 (0.0229)
Accumulating evaluation results...
DONE (t=0.18s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.030
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.113
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.005
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.031
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.137
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.278
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.278
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.278
*****Validation*****
Loss: 0.2094 | Classifier Loss: 0.1051 | Box Reg Loss: 0.0910 | RPN Box Reg Loss: 0.0062 | Objectness Loss: 0.0072
creating index...
index created!
Test:  [ 0/10]  eta: 0:00:27  model_time: 0.4156 (0.4156)  evaluator_time: 0.0123 (0.0123)  time: 2.7427  data: 2.2656  max mem: 9591
Test:  [ 9/10]  eta: 0:00:00  model_time: 0.4156 (0.3820)  evaluator_time: 0.0132 (0.0139)  time: 0.6623  data: 0.2270  max mem: 9591
Test: Total time: 0:00:06 (0.6705 s / it)
Averaged stats: model_time: 0.4156 (0.3820)  evaluator_time: 0.0132 (0.0139)
Accumulating evaluation results...
DONE (t=0.07s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.033
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.099
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.009
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.033
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.123
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.247
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.247
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.247
*****Training*****
MAP: 0.1128
*****Validationg*****
MAP: 0.0994


class MedInferenceDataset(Dataset):
    def __init__(self, root, transforms=None):
        """
        初始化資料集，只處理圖片檔案。

        :param root: 圖片資料夾路徑
        :param transforms: 圖片處理方法（可選）
        """
        self.root = root
        self.transforms = transforms
        self.image_files = sorted(
            [f for f in os.listdir(self.root) if f.lower().endswith(('.png', '.jpg', '.jpeg'))]
        )

    def get_image(self, index):
        """
        讀取指定索引的圖片。

        :param index: 圖片索引
        :return: PIL 圖片物件
        """
        image_path = os.path.join(self.root, self.image_files[index])
        image = Image.open(image_path).convert("RGB")

        return image

    def __getitem__(self, index):
        image = self.get_image(index)
        #print(f"Fetching image at index: {index}")
        if self.transforms:
            image = self.transforms(image)

        target = {
            'boxes': torch.zeros((0, 4), dtype=torch.float32),  # 空的 bounding box
            'labels': torch.tensor([], dtype=torch.int64),      # 空的標籤
            'image_id': torch.tensor([index]),                 # 圖片 ID
            'size': image.shape[1:]  # 圖片原始尺寸 (高度, 寬度)
        }
        #'size': image.size[::-1],  # 圖片原始尺寸 (高度, 寬度)
        #'size': image.shape[1:]
        #print(f"Image{image}")
        #print(f"Target{target}")
        return image, target

    def __len__(self):
        """
        返回資料集中圖片的數量。
        """
        return len(self.image_files)


def get_transform():

    transform = v2.Compose(
                [
                    v2.ToImage(), ## Used while using PIL image
                    #v2.ConvertBoundingBoxFormat(tv_tensors.BoundingBoxFormat.XYXY),
                    v2.ToDtype(torch.float32, scale=True),

                ])

    return transform


from torchvision.ops import nms
def predict(input_tensor, model, device, detection_threshold):
    outputs = model(input_tensor)
    pred_classes = [config.categories[i] for i in outputs[0]['labels'].cpu().numpy()]
    pred_labels = outputs[0]['labels'].cpu().numpy()
    pred_scores = outputs[0]['scores'].detach().cpu().numpy()
    pred_bboxes = outputs[0]['boxes'].detach().cpu().numpy()

    # 應用 NMS
    bboxes_tensor = torch.tensor(pred_bboxes, device=device)
    scores_tensor = torch.tensor(pred_scores, device=device)

    # NMS 返回保留的索引
    keep_indices = nms(bboxes_tensor, scores_tensor, iou_threshold=0.5)

    # 根據保留的索引過濾
    pred_bboxes = pred_bboxes[keep_indices.cpu().numpy()]
    pred_scores = pred_scores[keep_indices.cpu().numpy()]
    pred_labels = pred_labels[keep_indices.cpu().numpy()]
    pred_classes = [pred_classes[i] for i in keep_indices.cpu().numpy()]
    boxes, classes, labels, indices,scores = [], [], [], [], []
    for index in range(len(pred_scores)):
        if pred_scores[index] >= detection_threshold:
            boxes.append(pred_bboxes[index].astype(np.int32))
            classes.append(pred_classes[index])
            labels.append(pred_labels[index])
            indices.append(index)
            scores.append(pred_scores[index])
    boxes = np.int32(boxes)

    return boxes, classes, labels, indices, scores

COLORS = np.random.uniform(0, 255, size=(len(config.categories), 3))

def draw_boxes(boxes, labels, classes, image):
    for i, box in enumerate(boxes):
        # Convert RGB to BGR for OpenCV
        color = COLORS[labels[i]].astype(int)[::-1]

        # Draw the bounding box
        cv2.rectangle(
            image,
            (int(box[0]), int(box[1])),
            (int(box[2]), int(box[3])),
            color.tolist(), 8
        )

        # Increase font size and thickness for label
        font_scale = 4 # Increase this value for larger font
        thickness = 10     # Increase thickness for better visibility

        # Add class label as text
        cv2.putText(image, classes[i],
                    (int(box[0]), int(box[1]) - 10),  # Adjust text position
                    cv2.FONT_HERSHEY_SIMPLEX,
                    font_scale,
                    color.tolist(),
                    thickness,
                    lineType=cv2.LINE_AA)
    return image


def run_predictions(model, test_loader, device, detection_threshold):
    model.eval()  # 確保模型處於評估模式
    all_predictions = []

    with torch.no_grad():  # 關閉梯度計算
        for batch_idx, (images2, targets) in enumerate(test_loader):
                #print(f"========================batch_idx{batch_idx}=========================================")
            # 將每張圖片傳遞到 predict 函式
            #for i, image in enumerate(images):
                #print(f"================image{images2}================================")
                image_data = images2[0]  # 假設 images[0] 是我們的圖像
                input_tensor = image_data.unsqueeze(0).to(config.device)  # 增加 batch 維度並移至裝置
                #print(input_tensor)
                boxes, classes, labels, indices, scores = predict(
                    input_tensor, model, device, detection_threshold
                )


                # 組裝預測結果
                target_dict = targets[0]  # 假設第 0 項是目標數據字典
                #print(f"target_dict{target_dict}")
                img_id = target_dict["image_id"].item()
                img_size = target_dict["size"]  # 取得圖片原始大小，應該是 (height, width)

            # 取得圖片的寬度和高度
                img_height, img_width = img_size[0], img_size[1]

            # 正規化邊界框
                for j, box in enumerate(boxes):
                # 原始邊界框坐標 [xmin, ymin, xmax, ymax]
                    xmin, ymin, xmax, ymax = box

                # 根據圖片的寬度和高度進行正規化
                    xmin_norm = xmin / img_width
                    ymin_norm = ymin / img_height
                    xmax_norm = xmax / img_width
                    ymax_norm = ymax / img_height

                # 存儲正規化後的預測結果
                    all_predictions.append({
                        "ID": img_id,
                        "category": classes[j],
                        "score": scores[j],
                        "xmin": xmin_norm,
                        "xmax": xmax_norm,
                        "ymin": ymin_norm,
                        "ymax": ymax_norm,
                    })

    return all_predictions


# 測試資料集與 DataLoader
test_dataset = MedInferenceDataset(root=config.test_root, transforms=get_transform())
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False,num_workers=config.workers, collate_fn=collate_fn)
#test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)
model = fasterrcnn(num_classes = config.num_classes)
state_dict = torch.load("/content/drive/MyDrive/working/final.pth")
model.load_state_dict(state_dict["model"])
model.to(config.device)

FasterRCNN(
  (transform): GeneralizedRCNNTransform(
      Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
      Resize(min_size=(800,), max_size=1333, mode='bilinear')
  )
  (backbone): BackboneWithFPN(
    (body): IntermediateLayerGetter(
      (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
      (bn1): FrozenBatchNorm2d(64, eps=0.0)
      (relu): ReLU(inplace=True)
      (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
      (layer1): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): FrozenBatchNorm2d(256, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer2): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(512, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (3): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer3): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(1024, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (3): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (4): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (5): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer4): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(2048, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
    )
    (fpn): FeaturePyramidNetwork(
      (inner_blocks): ModuleList(
        (0): Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (1): Conv2dNormActivation(
          (0): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (2): Conv2dNormActivation(
          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (3): Conv2dNormActivation(
          (0): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))
        )
      )
      (layer_blocks): ModuleList(
        (0-3): 4 x Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        )
      )
      (extra_blocks): LastLevelMaxPool()
    )
  )
  (rpn): RegionProposalNetwork(
    (anchor_generator): AnchorGenerator()
    (head): RPNHead(
      (conv): Sequential(
        (0): Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
          (1): ReLU(inplace=True)
        )
      )
      (cls_logits): Conv2d(256, 3, kernel_size=(1, 1), stride=(1, 1))
      (bbox_pred): Conv2d(256, 12, kernel_size=(1, 1), stride=(1, 1))
    )
  )
  (roi_heads): RoIHeads(
    (box_roi_pool): MultiScaleRoIAlign(featmap_names=['0', '1', '2', '3'], output_size=(7, 7), sampling_ratio=2)
    (box_head): TwoMLPHead(
      (fc6): Linear(in_features=12544, out_features=1024, bias=True)
      (fc7): Linear(in_features=1024, out_features=1024, bias=True)
    )
    (box_predictor): FastRCNNPredictor(
      (cls_score): Linear(in_features=1024, out_features=8, bias=True)
      (bbox_pred): Linear(in_features=1024, out_features=32, bias=True)
    )
  )
)


predictions = run_predictions(model, test_loader, config.device,0.5)
# 建立 DataFrame 並根據 test_ids 排序
results_df = pd.DataFrame(predictions)
results_df["ID"] = results_df["ID"].astype(int) + 1
results_df = results_df.sort_values(by="ID")
test_csv_df = pd.read_csv('/content/drive/MyDrive/hwk05_data/test.csv')
test_csv_df['filename_id'] = test_csv_df['Filename'].str.extract(r'(\d+)').astype(int)
merged_df = results_df.merge(test_csv_df[['filename_id', 'ID']], left_on='ID', right_on='filename_id', how='left')
# 可以刪除不再需要的欄位
merged_df = merged_df.drop(columns=['filename_id', 'ID_x'])
# 將 ID_y 移到第一列並將名稱更改為 ID
merged_df = merged_df.rename(columns={'ID_y': 'ID'})  # 重命名 ID_y 為 ID
# 將 ID 列移動到最前面
merged_df = merged_df[['ID'] + [col for col in merged_df.columns if col != 'ID']]
merged_df.head(10)


merged_df.tail(10)


merged_df.shape

(368, 7)


from google.colab import files
# 保存 CSV 文件到指定路徑
merged_df.to_csv('/content/SGD_batch8.csv', index=False, header = True)
# 下載文件到本地機器
files.download('/content/SGD_batch8.csv')


def plot_ablation_cam_images(transforms, model):

    rows, cols = 4, 2
    fig = plt.figure(figsize = (10, 20))
    grid = plt.GridSpec(rows, cols)
    best_ckpt = torch.load("/content/drive/MyDrive/working/final.pth", map_location = config.device)
    #best_ckpt = torch.load("/kaggle/input/hw05final/pytorch/default/1/coco_t3.pth", map_location = config.device)
    model.load_state_dict(best_ckpt["model"])
    model.eval().to(config.device)
    target_layers = [model.backbone]

    cam = AblationCAM(model,
                      target_layers,
                      reshape_transform = fasterrcnn_reshape_transform,
                      ablation_layer = AblationLayerFasterRCNN(),
                      ratio_channels_to_ablate = 1.0)

    for i in range(rows * cols):

        all_images = os.listdir(os.path.join(config.root, config.categories[i]))
        image_path = os.path.join(config.root, config.categories[i], all_images[0])
        image = Image.open(image_path).convert("RGB")
        input_tensor = transforms(image)
        input_tensor = input_tensor.to(config.device)
        input_tensor = input_tensor.unsqueeze(0)
        image = np.array(image)
        image_float_np = np.float32(image) / 255

        boxes, classes, labels, indices, scores = predict(input_tensor, model, config.device, 0.5)
        image = draw_boxes(boxes, labels, classes, image)
        targets = [FasterRCNNBoxScoreTarget(labels = labels, bounding_boxes = boxes)]

        grayscale_cam = cam(input_tensor, targets = targets)
        grayscale_cam = grayscale_cam[0, :]
        cam_image = show_cam_on_image(image_float_np, grayscale_cam, use_rgb = True)
        image_with_bounding_boxes = draw_boxes(boxes, labels, classes, cam_image)

        categories = fig.add_subplot(grid[i])
        categories.set_axis_off()

        gs = gridspec.GridSpecFromSubplotSpec(1, 1, subplot_spec = grid[i])

        ax = fig.add_subplot(gs[0])
        ax.imshow(image_with_bounding_boxes)
        ax.set_title(f"{config.categories[i]}")
        ax.axis("off")

    fig.patch.set_facecolor('white')
    fig.suptitle("AblationCAM Images of 8 categories\n", fontweight = 'bold', size = 16)
    fig.tight_layout()


plot_ablation_cam_images(transforms = get_transform(), model = fasterrcnn(config.num_classes))

100%|██████████| 40/40 [00:20<00:00,  1.99it/s]
100%|██████████| 40/40 [00:23<00:00,  1.71it/s]
100%|██████████| 40/40 [00:23<00:00,  1.69it/s]
100%|██████████| 40/40 [00:22<00:00,  1.80it/s]
100%|██████████| 40/40 [00:22<00:00,  1.80it/s]
100%|██████████| 40/40 [00:23<00:00,  1.74it/s]
100%|██████████| 40/40 [00:22<00:00,  1.82it/s]
100%|██████████| 40/40 [00:21<00:00,  1.83it/s]


import matplotlib.pyplot as plt
import matplotlib.patches as patches
import pandas as pd
import os
from PIL import Image
from matplotlib.colors import ListedColormap
from matplotlib.colors import TABLEAU_COLORS


def plot_image_with_boxes_test(df, test_csv_path, id_column, bbox_columns, image_dir):
    """
    Plots images with bounding boxes for each unique ID in the dataframe, with all boxes for the same ID on one image.

    Args:
        df (pd.DataFrame): DataFrame containing bounding box information.
        test_csv_path (str): Path to the test.csv file.
        id_column (str): Column name in df corresponding to the ID.
        bbox_columns (list of str): Column names for bounding box coordinates ["xmin", "xmax", "ymin", "ymax"].
        image_dir (str): Path to the directory containing processed test images.
        scale (float): Scale factor for resizing the image (0 < scale <= 1).
    """
    # Load test.csv
    test_csv = pd.read_csv("/content/drive/MyDrive/processed_images/test.csv")

    scale = 0.2
    unique_ids = df[id_column].unique()
    num_images = len(unique_ids)

    # Create subplots
    fig, axes = plt.subplots(
        nrows=(num_images + 2) // 3, ncols=3, figsize=(15, 5 * ((num_images + 2) // 3))
    )
    axes = axes.flatten()

    # Create a color map for categories
    categories = df['category'].unique()
    category_colors = {category: plt.cm.get_cmap('tab10')(i % 10) for i, category in enumerate(categories)}

    for ax, img_id in zip(axes, unique_ids):
        img_data = df[df[id_column] == img_id]

        # Find the corresponding dcm name in test.csv
        dcm_name = test_csv.loc[test_csv['ID'] == img_id, "Filename"].values[0]
        dcm_name = os.path.splitext(dcm_name)[0]

        # Construct the image path
        image_path = os.path.join(image_dir, f"{dcm_name}.jpg")

        # Check if the image file exists
        if not os.path.exists(image_path):
            ax.set_title(f"Image {dcm_name} not found.")
            ax.axis('off')
            continue

        # Open the image
        image = Image.open(image_path)
        width, height = image.size

        # Resize the image based on scale
        image = image.resize((int(width * scale), int(height * scale)))
        width, height = image.size

        # Plot the image
        ax.imshow(image)

        # Draw bounding boxes
        for i, row in img_data.iterrows():
            xmin, xmax, ymin, ymax = [row[col] for col in bbox_columns]
            bounding_width = (xmax - xmin) * width
            bounding_height = (ymax - ymin) * height

            # Get the color for the category
            category = row.get("category", "Unknown")
            color = category_colors.get(category, 'gray')  # Default to gray if category is not in the map

            rect = patches.Rectangle(
                (xmin * width, ymin * height),
                bounding_width,
                bounding_height,
                linewidth=2,
                edgecolor=color,
                facecolor='none'
            )
            ax.add_patch(rect)

            # Display the category text near the bounding box
            ax.text(
                xmin * width,
                ymin * height - 10,
                category,
                color='white',
                fontsize=8,
                bbox=dict(facecolor=color, alpha=0.5)
            )

        ax.set_title(f"ID: {img_id}")
        ax.axis('off')

    # Turn off any unused subplots
    for ax in axes[num_images:]:
        ax.axis('off')

    plt.tight_layout()
    plt.show()


# Example usage
plot_image_with_boxes_test(merged_df[:10], "test.csv", "ID", ["xmin", "xmax", "ymin", "ymax"], "/content/drive/MyDrive/processed_images/test")


def train_one_epoch(model, train_loader, optimizer, lr_scheduler,  epoch, device):
    model.train()

    train_loss = []
    train_loss_dict = []

    #lr_scheduler = None

    for images, targets in tqdm(train_loader):
        images = [image.to(device) for image in images]
        targets = [{k: (torch.tensor(v,device=device) if not isinstance(v, torch.Tensor) else v.to(device)) for k, v in t.items()} for t in targets]

        loss_dict = model(images, targets)
        losses = sum(loss for loss in loss_dict.values())

        batch_loss_value = losses.item()
        batch_loss_dict = {k: v.item() for k, v in loss_dict.items()}

        train_loss.append(batch_loss_value)
        train_loss_dict.append(batch_loss_dict)

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()

    train_loss = np.mean(train_loss)
    train_loss_dict = pd.DataFrame(train_loss_dict).mean()
    train_loss_classifier = train_loss_dict.loss_classifier
    train_loss_box_reg = train_loss_dict.loss_box_reg
    train_loss_rpn_box_reg = train_loss_dict.loss_rpn_box_reg
    train_loss_objectness = train_loss_dict.loss_objectness

    return train_loss, train_loss_classifier, train_loss_box_reg, train_loss_rpn_box_reg, train_loss_objectness


##AdamW OneSample
def main():

    seed_everything(config.seed)
    g = torch.Generator()
    g.manual_seed(config.seed)

    train_dataset = MedDataset(root = config.root, info_root = config.info_root, split = "train", transforms = medTransform(train=True))
    val_dataset = MedDataset(root = config.root, info_root = config.info_root, split = "val",  transforms = medTransform(train=False))

    train_loader = DataLoader(train_dataset, batch_size = config.batch_size, shuffle = True,
                              num_workers=config.workers, collate_fn = collate_fn,pin_memory=True
                             )
    val_loader = DataLoader(val_dataset, batch_size = config.batch_size, shuffle = False,
                            num_workers=config.workers, worker_init_fn=seed_worker,
                            generator=g, collate_fn = collate_fn,pin_memory=True
                           )


    device = config.device
    model =  fasterrcnn(num_classes = config.num_classes)
    model.to(device)

    parameters = [p for p in model.parameters() if p.requires_grad]

    # Use AdamW optimizer
    optimizer = torch.optim.AdamW(parameters, lr=config.lr, weight_decay=config.weight_decay)
    # Use OneSampleLR scheduler
    scheduler = torch.optim.lr_scheduler.OneCycleLR(
    optimizer,
    max_lr=config.lr,  # Maximum learning rate
    steps_per_epoch=len(train_loader),  # Number of steps per epoch
    epochs=config.epochs,  # Total number of epochs
    anneal_strategy='cos',  # Cosine annealing for learning rate decay
    pct_start=0.3  # Proportion of the cycle spent increasing learning rate
)
    best_val_loss = float("inf")
    best_map50 = 0.0
    history = {
        "train": {
            "loss": [],
            "loss_classifier": [],
            "loss_box_reg": [],
            "loss_rpn_box_reg": [],
            "loss_objectness": []
        },
        "val": {
            "loss": [],
            "loss_classifier": [],
            "loss_box_reg": [],
            "loss_rpn_box_reg": [],
            "loss_objectness": []
        },
        "map50":{
            "train":[],
            "valid":[],
        }
    }
    best_idx = 0
    print('start')
    for epoch in range(config.epochs):
        print()
        train_loss, train_loss_classifier, train_loss_box_reg, train_loss_rpn_box_reg, train_loss_objectness = train_one_epoch(
            model, train_loader, optimizer, scheduler, epoch, device,
        )

        val_loss, val_loss_classifier, val_loss_box_reg, val_loss_rpn_box_reg, val_loss_objectness = validation(
            model, val_loader, device
        )

        ## Training
        history["train"]["loss"].append(train_loss)
        history["train"]["loss_classifier"].append(train_loss_classifier)
        history["train"]["loss_box_reg"].append(train_loss_box_reg)
        history["train"]["loss_rpn_box_reg"].append(train_loss_rpn_box_reg)
        history["train"]["loss_objectness"].append(train_loss_objectness)
        ## Validation
        history["val"]["loss"].append(val_loss)
        history["val"]["loss_classifier"].append(val_loss_classifier)
        history["val"]["loss_box_reg"].append(val_loss_box_reg)
        history["val"]["loss_rpn_box_reg"].append(val_loss_rpn_box_reg)
        history["val"]["loss_objectness"].append(val_loss_objectness)


        print(f'Epoch: {epoch+1}/{config.epochs} | LR: {optimizer.state_dict()["param_groups"][0]["lr"]:.6f}')

        print("*****Training*****")
        print(f'Loss: {train_loss:.4f} | Classifier Loss: {train_loss_classifier:.4f} | Box Reg Loss: {train_loss_box_reg:.4f} | RPN Box Reg Loss: {train_loss_rpn_box_reg:.4f} | Objectness Loss: {train_loss_objectness:.4f}')
        train_evaluator = evaluate(model, train_loader, device = device)
        print("*****Validation*****")
        print(f'Loss: {val_loss:.4f} | Classifier Loss: {val_loss_classifier:.4f} | Box Reg Loss: {val_loss_box_reg:.4f} | RPN Box Reg Loss: {val_loss_rpn_box_reg:.4f} | Objectness Loss: {val_loss_objectness:.4f}')
        valid_evaluator = evaluate(model, val_loader, device = device)

        train_map50 = train_evaluator.coco_eval['bbox'].stats[1]
        valid_map50 = valid_evaluator.coco_eval['bbox'].stats[1]
        print("*****Training*****")
        print(f'MAP: {train_map50:.4f}')
        print("*****Validationg*****")
        print(f'MAP: {valid_map50:.4f}')
        history["map50"]["train"].append(train_map50)
        history["map50"]["valid"].append(valid_map50)

        ## TODO save your model

        if valid_map50 > best_map50:
            save_file = {
                "model": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "epoch": epoch,
                "args": config
            }
            best_idx=epoch
            best_map50 = valid_map50
            torch.save(save_file, os.path.join(config.save_root,"final.pth"))
        #scheduler.step()
    print(f'Best epoch in {best_idx+1}')


    ## Evaluation result
    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["map50"]["train"], label = 'Training map50')
    plt.plot(range(config.epochs), history["map50"]["valid"], label = 'Validation map50')
    plt.xlabel('Epoch')
    plt.ylabel('map')
    plt.legend()
    plt.title('Training and Validation map50')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss"], label = 'Training Loss')
    plt.plot(range(config.epochs), history["val"]["loss"], label = 'Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title('Training and Validation Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_classifier"], label = 'Training Classifier Loss')
    plt.plot(range(config.epochs), history["val"]["loss_classifier"], label = 'Validation Classifier Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Classifier Loss')
    plt.legend()
    plt.title('Training and Validation Classifier Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_box_reg"], label = 'Training Box Reg Loss')
    plt.plot(range(config.epochs), history["val"]["loss_box_reg"], label = 'Validation Box Reg Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Box Reg Loss')
    plt.legend()
    plt.title('Training and Validation Box Reg Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_rpn_box_reg"], label = 'Training RPN Box Reg Loss')
    plt.plot(range(config.epochs), history["val"]["loss_rpn_box_reg"], label = 'Validation RPN Box Reg Loss')

    plt.xlabel('Epoch')
    plt.ylabel('RPN Box Reg Loss')
    plt.legend()
    plt.title('Training and Validation RPN Box Reg Loss Curves')
    plt.show()

    plt.figure(figsize = (12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(range(config.epochs), history["train"]["loss_objectness"], label = 'Training Objectness Loss')
    plt.plot(range(config.epochs), history["val"]["loss_objectness"], label = 'Validation Objectness Loss')

    plt.xlabel('Epoch')
    plt.ylabel('Objectness Loss')
    plt.legend()
    plt.title('Training and Validation Objectness Loss Curves')
    plt.show()


## IMAGENET 3
if __name__ == "__main__":
    main()

loading annotations into memory...
Done (t=0.00s)
creating index...
index created!
loading annotations into memory...
Done (t=0.00s)
creating index...
index created!
start

  0%|          | 0/69 [00:00<?, ?it/s]

  0%|          | 0/19 [00:00<?, ?it/s]

Epoch: 1/20 | LR: 0.000105
*****Training*****
Loss: 0.3027 | Classifier Loss: 0.1905 | Box Reg Loss: 0.0883 | RPN Box Reg Loss: 0.0087 | Objectness Loss: 0.0152
creating index...
index created!
Test:  [ 0/69]  eta: 0:01:29  model_time: 0.2271 (0.2271)  evaluator_time: 0.0164 (0.0164)  time: 1.2908  data: 1.0239  max mem: 9908
Test:  [68/69]  eta: 0:00:00  model_time: 0.1873 (0.1963)  evaluator_time: 0.0070 (0.0084)  time: 0.2180  data: 0.0003  max mem: 9908
Test: Total time: 0:00:16 (0.2461 s / it)
Averaged stats: model_time: 0.1873 (0.1963)  evaluator_time: 0.0070 (0.0084)
Accumulating evaluation results...
DONE (t=0.18s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.081
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.208
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.056
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.081
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.302
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.368
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.368
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.368
*****Validation*****
Loss: 0.1858 | Classifier Loss: 0.0938 | Box Reg Loss: 0.0802 | RPN Box Reg Loss: 0.0062 | Objectness Loss: 0.0055
creating index...
index created!
Test:  [ 0/19]  eta: 0:00:28  model_time: 0.1936 (0.1936)  evaluator_time: 0.0074 (0.0074)  time: 1.5029  data: 1.2771  max mem: 9908
Test:  [18/19]  eta: 0:00:00  model_time: 0.1936 (0.1885)  evaluator_time: 0.0074 (0.0071)  time: 0.2841  data: 0.0676  max mem: 9908
Test: Total time: 0:00:05 (0.2889 s / it)
Averaged stats: model_time: 0.1936 (0.1885)  evaluator_time: 0.0074 (0.0071)
Accumulating evaluation results...
DONE (t=0.06s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.078
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.186
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.053
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.078
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.277
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.334
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.334
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.334
*****Training*****
MAP: 0.2082
*****Validationg*****
MAP: 0.1862

  0%|          | 0/69 [00:00<?, ?it/s]

  0%|          | 0/19 [00:00<?, ?it/s]

Epoch: 2/20 | LR: 0.000281
*****Training*****
Loss: 0.1845 | Classifier Loss: 0.0891 | Box Reg Loss: 0.0815 | RPN Box Reg Loss: 0.0072 | Objectness Loss: 0.0066
creating index...
index created!
Test:  [ 0/69]  eta: 0:01:35  model_time: 0.2119 (0.2119)  evaluator_time: 0.0241 (0.0241)  time: 1.3803  data: 1.1206  max mem: 9908
Test:  [68/69]  eta: 0:00:00  model_time: 0.1952 (0.1951)  evaluator_time: 0.0074 (0.0084)  time: 0.2212  data: 0.0003  max mem: 9908
Test: Total time: 0:00:16 (0.2459 s / it)
Averaged stats: model_time: 0.1952 (0.1951)  evaluator_time: 0.0074 (0.0084)
Accumulating evaluation results...
DONE (t=0.19s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.136
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.292
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.114
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.136
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.470
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.506
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.506
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.506
*****Validation*****
Loss: 0.1719 | Classifier Loss: 0.0896 | Box Reg Loss: 0.0716 | RPN Box Reg Loss: 0.0060 | Objectness Loss: 0.0047
creating index...
index created!
Test:  [ 0/19]  eta: 0:00:26  model_time: 0.1981 (0.1981)  evaluator_time: 0.0123 (0.0123)  time: 1.4009  data: 1.1652  max mem: 9908
Test:  [18/19]  eta: 0:00:00  model_time: 0.1944 (0.1889)  evaluator_time: 0.0092 (0.0092)  time: 0.2859  data: 0.0669  max mem: 9908
Test: Total time: 0:00:05 (0.2911 s / it)
Averaged stats: model_time: 0.1944 (0.1889)  evaluator_time: 0.0092 (0.0092)
Accumulating evaluation results...
DONE (t=0.06s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.131
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.304
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.099
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.131
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.444
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.468
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.468
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.468
*****Training*****
MAP: 0.2919
*****Validationg*****
MAP: 0.3045

  0%|          | 0/69 [00:00<?, ?it/s]

  0%|          | 0/19 [00:00<?, ?it/s]

Epoch: 3/20 | LR: 0.000522
*****Training*****
Loss: 0.2108 | Classifier Loss: 0.1010 | Box Reg Loss: 0.0781 | RPN Box Reg Loss: 0.0096 | Objectness Loss: 0.0221
creating index...
index created!
Test:  [ 0/69]  eta: 0:01:53  model_time: 0.2093 (0.2093)  evaluator_time: 0.0136 (0.0136)  time: 1.6457  data: 1.3981  max mem: 9908
Test:  [68/69]  eta: 0:00:00  model_time: 0.1942 (0.1959)  evaluator_time: 0.0089 (0.0096)  time: 0.2260  data: 0.0004  max mem: 9908
Test: Total time: 0:00:17 (0.2515 s / it)
Averaged stats: model_time: 0.1942 (0.1959)  evaluator_time: 0.0089 (0.0096)
Accumulating evaluation results...
DONE (t=0.23s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.046
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.134
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.022
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.046
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.192
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.303
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.308
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.308
*****Validation*****
Loss: 0.1633 | Classifier Loss: 0.0741 | Box Reg Loss: 0.0633 | RPN Box Reg Loss: 0.0113 | Objectness Loss: 0.0146
creating index...
index created!
Test:  [ 0/19]  eta: 0:00:31  model_time: 0.2001 (0.2001)  evaluator_time: 0.0111 (0.0111)  time: 1.6517  data: 1.4160  max mem: 9908
Test:  [18/19]  eta: 0:00:00  model_time: 0.1954 (0.1901)  evaluator_time: 0.0098 (0.0104)  time: 0.2962  data: 0.0749  max mem: 9908
Test: Total time: 0:00:05 (0.3015 s / it)
Averaged stats: model_time: 0.1954 (0.1901)  evaluator_time: 0.0098 (0.0104)
Accumulating evaluation results...
DONE (t=0.07s).
IoU metric: bbox
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.048
 Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.142
 Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.026
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.048
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.201
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.315
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.315
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000
 Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.315
*****Training*****
MAP: 0.1343
*****Validationg*****
MAP: 0.1424


class MedInferenceDataset(Dataset):
    def __init__(self, root, transforms=None):
        """
        初始化資料集，只處理圖片檔案。

        :param root: 圖片資料夾路徑
        :param transforms: 圖片處理方法（可選）
        """
        self.root = root
        self.transforms = transforms
        self.image_files = sorted(
            [f for f in os.listdir(self.root) if f.lower().endswith(('.png', '.jpg', '.jpeg'))]
        )

    def get_image(self, index):
        """
        讀取指定索引的圖片。

        :param index: 圖片索引
        :return: PIL 圖片物件
        """
        image_path = os.path.join(self.root, self.image_files[index])
        image = Image.open(image_path).convert("RGB")

        return image

    def __getitem__(self, index):
        image = self.get_image(index)
        #print(f"Fetching image at index: {index}")
        if self.transforms:
            image = self.transforms(image)

        target = {
            'boxes': torch.zeros((0, 4), dtype=torch.float32),  # 空的 bounding box
            'labels': torch.tensor([], dtype=torch.int64),      # 空的標籤
            'image_id': torch.tensor([index]),                 # 圖片 ID
            'size': image.shape[1:]  # 圖片原始尺寸 (高度, 寬度)
        }
        #'size': image.size[::-1],  # 圖片原始尺寸 (高度, 寬度)
        #'size': image.shape[1:]
        #print(f"Image{image}")
        #print(f"Target{target}")
        return image, target

    def __len__(self):
        """
        返回資料集中圖片的數量。
        """
        return len(self.image_files)


def get_transform():

    transform = v2.Compose(
                [
                    v2.ToImage(), ## Used while using PIL image
                    #v2.ConvertBoundingBoxFormat(tv_tensors.BoundingBoxFormat.XYXY),
                    v2.ToDtype(torch.float32, scale=True),

                ])

    return transform


from torchvision.ops import nms
def predict(input_tensor, model, device, detection_threshold):
    outputs = model(input_tensor)
    pred_classes = [config.categories[i] for i in outputs[0]['labels'].cpu().numpy()]
    pred_labels = outputs[0]['labels'].cpu().numpy()
    pred_scores = outputs[0]['scores'].detach().cpu().numpy()
    pred_bboxes = outputs[0]['boxes'].detach().cpu().numpy()

    # 應用 NMS
    bboxes_tensor = torch.tensor(pred_bboxes, device=device)
    scores_tensor = torch.tensor(pred_scores, device=device)

    # NMS 返回保留的索引
    keep_indices = nms(bboxes_tensor, scores_tensor, iou_threshold=0.5)

    # 根據保留的索引過濾
    pred_bboxes = pred_bboxes[keep_indices.cpu().numpy()]
    pred_scores = pred_scores[keep_indices.cpu().numpy()]
    pred_labels = pred_labels[keep_indices.cpu().numpy()]
    pred_classes = [pred_classes[i] for i in keep_indices.cpu().numpy()]
    boxes, classes, labels, indices,scores = [], [], [], [], []
    for index in range(len(pred_scores)):
        if pred_scores[index] >= detection_threshold:
            boxes.append(pred_bboxes[index].astype(np.int32))
            classes.append(pred_classes[index])
            labels.append(pred_labels[index])
            indices.append(index)
            scores.append(pred_scores[index])
    boxes = np.int32(boxes)

    return boxes, classes, labels, indices, scores

COLORS = np.random.uniform(0, 255, size=(len(config.categories), 3))

def draw_boxes(boxes, labels, classes, image):
    for i, box in enumerate(boxes):
        # Convert RGB to BGR for OpenCV
        color = COLORS[labels[i]].astype(int)[::-1]

        # Draw the bounding box
        cv2.rectangle(
            image,
            (int(box[0]), int(box[1])),
            (int(box[2]), int(box[3])),
            color.tolist(), 8
        )

        # Increase font size and thickness for label
        font_scale = 4 # Increase this value for larger font
        thickness = 10     # Increase thickness for better visibility

        # Add class label as text
        cv2.putText(image, classes[i],
                    (int(box[0]), int(box[1]) - 10),  # Adjust text position
                    cv2.FONT_HERSHEY_SIMPLEX,
                    font_scale,
                    color.tolist(),
                    thickness,
                    lineType=cv2.LINE_AA)
    return image


def run_predictions(model, test_loader, device, detection_threshold):
    model.eval()  # 確保模型處於評估模式
    all_predictions = []

    with torch.no_grad():  # 關閉梯度計算
        for batch_idx, (images2, targets) in enumerate(test_loader):
                #print(f"========================batch_idx{batch_idx}=========================================")
            # 將每張圖片傳遞到 predict 函式
            #for i, image in enumerate(images):
                #print(f"================image{images2}================================")
                image_data = images2[0]  # 假設 images[0] 是我們的圖像
                input_tensor = image_data.unsqueeze(0).to(config.device)  # 增加 batch 維度並移至裝置
                #print(input_tensor)
                boxes, classes, labels, indices, scores = predict(
                    input_tensor, model, device, detection_threshold
                )


                # 組裝預測結果
                target_dict = targets[0]  # 假設第 0 項是目標數據字典
                #print(f"target_dict{target_dict}")
                img_id = target_dict["image_id"].item()
                img_size = target_dict["size"]  # 取得圖片原始大小，應該是 (height, width)

            # 取得圖片的寬度和高度
                img_height, img_width = img_size[0], img_size[1]

            # 正規化邊界框
                for j, box in enumerate(boxes):
                # 原始邊界框坐標 [xmin, ymin, xmax, ymax]
                    xmin, ymin, xmax, ymax = box

                # 根據圖片的寬度和高度進行正規化
                    xmin_norm = xmin / img_width
                    ymin_norm = ymin / img_height
                    xmax_norm = xmax / img_width
                    ymax_norm = ymax / img_height

                # 存儲正規化後的預測結果
                    all_predictions.append({
                        "ID": img_id,
                        "category": classes[j],
                        "score": scores[j],
                        "xmin": xmin_norm,
                        "xmax": xmax_norm,
                        "ymin": ymin_norm,
                        "ymax": ymax_norm,
                    })

    return all_predictions


# 測試資料集與 DataLoader
test_dataset = MedInferenceDataset(root=config.test_root, transforms=get_transform())
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False,num_workers=config.workers, collate_fn=collate_fn)
#test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)
model = fasterrcnn(num_classes = config.num_classes)
state_dict = torch.load("/content/drive/MyDrive/working/final.pth")
model.load_state_dict(state_dict["model"])
model.to(config.device)

FasterRCNN(
  (transform): GeneralizedRCNNTransform(
      Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
      Resize(min_size=(800,), max_size=1333, mode='bilinear')
  )
  (backbone): BackboneWithFPN(
    (body): IntermediateLayerGetter(
      (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
      (bn1): FrozenBatchNorm2d(64, eps=0.0)
      (relu): ReLU(inplace=True)
      (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
      (layer1): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): FrozenBatchNorm2d(256, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(64, eps=0.0)
          (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(64, eps=0.0)
          (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(256, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer2): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(512, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (3): Bottleneck(
          (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(128, eps=0.0)
          (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(128, eps=0.0)
          (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(512, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer3): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(1024, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (3): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (4): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (5): Bottleneck(
          (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(256, eps=0.0)
          (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(256, eps=0.0)
          (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(1024, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
      (layer4): Sequential(
        (0): Bottleneck(
          (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
          (downsample): Sequential(
            (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)
            (1): FrozenBatchNorm2d(2048, eps=0.0)
          )
        )
        (1): Bottleneck(
          (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
        )
        (2): Bottleneck(
          (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn1): FrozenBatchNorm2d(512, eps=0.0)
          (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (bn2): FrozenBatchNorm2d(512, eps=0.0)
          (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
          (bn3): FrozenBatchNorm2d(2048, eps=0.0)
          (relu): ReLU(inplace=True)
        )
      )
    )
    (fpn): FeaturePyramidNetwork(
      (inner_blocks): ModuleList(
        (0): Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (1): Conv2dNormActivation(
          (0): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (2): Conv2dNormActivation(
          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))
        )
        (3): Conv2dNormActivation(
          (0): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))
        )
      )
      (layer_blocks): ModuleList(
        (0-3): 4 x Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        )
      )
      (extra_blocks): LastLevelMaxPool()
    )
  )
  (rpn): RegionProposalNetwork(
    (anchor_generator): AnchorGenerator()
    (head): RPNHead(
      (conv): Sequential(
        (0): Conv2dNormActivation(
          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
          (1): ReLU(inplace=True)
        )
      )
      (cls_logits): Conv2d(256, 3, kernel_size=(1, 1), stride=(1, 1))
      (bbox_pred): Conv2d(256, 12, kernel_size=(1, 1), stride=(1, 1))
    )
  )
  (roi_heads): RoIHeads(
    (box_roi_pool): MultiScaleRoIAlign(featmap_names=['0', '1', '2', '3'], output_size=(7, 7), sampling_ratio=2)
    (box_head): TwoMLPHead(
      (fc6): Linear(in_features=12544, out_features=1024, bias=True)
      (fc7): Linear(in_features=1024, out_features=1024, bias=True)
    )
    (box_predictor): FastRCNNPredictor(
      (cls_score): Linear(in_features=1024, out_features=8, bias=True)
      (bbox_pred): Linear(in_features=1024, out_features=32, bias=True)
    )
  )
)


predictions = run_predictions(model, test_loader, config.device,0.5)
# 建立 DataFrame 並根據 test_ids 排序
results_df = pd.DataFrame(predictions)
results_df["ID"] = results_df["ID"].astype(int) + 1
results_df = results_df.sort_values(by="ID")
#results_df.head(10)


test_csv_df = pd.read_csv('/content/drive/MyDrive/hwk05_data/test.csv')
test_csv_df['filename_id'] = test_csv_df['Filename'].str.extract(r'(\d+)').astype(int)


merged_df = results_df.merge(test_csv_df[['filename_id', 'ID']], left_on='ID', right_on='filename_id', how='left')
# 可以刪除不再需要的欄位
merged_df = merged_df.drop(columns=['filename_id', 'ID_x'])
# 將 ID_y 移到第一列並將名稱更改為 ID
merged_df = merged_df.rename(columns={'ID_y': 'ID'})  # 重命名 ID_y 為 ID
# 將 ID 列移動到最前面
merged_df = merged_df[['ID'] + [col for col in merged_df.columns if col != 'ID']]
merged_df.head(10)


merged_df.tail(10)


merged_df.shape

(227, 7)


from google.colab import files
# 保存 CSV 文件到指定路徑
merged_df.to_csv('/content/AdamWOneCyclelr10-3_batch4.csv', index=False, header = True)
# 下載文件到本地機器
files.download('/content/AdamWOneCyclelr10-3_batch4.csv')


def plot_ablation_cam_images(transforms, model):

    rows, cols = 4, 2
    fig = plt.figure(figsize = (10, 20))
    grid = plt.GridSpec(rows, cols)
    best_ckpt = torch.load("/content/drive/MyDrive/working/final.pth", map_location = config.device)
    #best_ckpt = torch.load("/kaggle/input/hw05final/pytorch/default/1/coco_t3.pth", map_location = config.device)
    model.load_state_dict(best_ckpt["model"])
    model.eval().to(config.device)
    target_layers = [model.backbone]

    cam = AblationCAM(model,
                      target_layers,
                      reshape_transform = fasterrcnn_reshape_transform,
                      ablation_layer = AblationLayerFasterRCNN(),
                      ratio_channels_to_ablate = 1.0)

    for i in range(rows * cols):

        all_images = os.listdir(os.path.join(config.root, config.categories[i]))
        image_path = os.path.join(config.root, config.categories[i], all_images[0])
        image = Image.open(image_path).convert("RGB")
        input_tensor = transforms(image)
        input_tensor = input_tensor.to(config.device)
        input_tensor = input_tensor.unsqueeze(0)
        image = np.array(image)
        image_float_np = np.float32(image) / 255

        boxes, classes, labels, indices, scores = predict(input_tensor, model, config.device, 0.5)
        image = draw_boxes(boxes, labels, classes, image)
        targets = [FasterRCNNBoxScoreTarget(labels = labels, bounding_boxes = boxes)]

        grayscale_cam = cam(input_tensor, targets = targets)
        grayscale_cam = grayscale_cam[0, :]
        cam_image = show_cam_on_image(image_float_np, grayscale_cam, use_rgb = True)
        image_with_bounding_boxes = draw_boxes(boxes, labels, classes, cam_image)

        categories = fig.add_subplot(grid[i])
        categories.set_axis_off()

        gs = gridspec.GridSpecFromSubplotSpec(1, 1, subplot_spec = grid[i])

        ax = fig.add_subplot(gs[0])
        ax.imshow(image_with_bounding_boxes)
        ax.set_title(f"{config.categories[i]}")
        ax.axis("off")

    fig.patch.set_facecolor('white')
    fig.suptitle("AblationCAM Images of 8 categories\n", fontweight = 'bold', size = 16)
    fig.tight_layout()


plot_ablation_cam_images(transforms = get_transform(), model = fasterrcnn(config.num_classes))

100%|██████████| 40/40 [00:20<00:00,  1.91it/s]
100%|██████████| 40/40 [00:21<00:00,  1.83it/s]
100%|██████████| 40/40 [00:22<00:00,  1.74it/s]
100%|██████████| 40/40 [00:23<00:00,  1.73it/s]
100%|██████████| 40/40 [00:22<00:00,  1.77it/s]
100%|██████████| 40/40 [00:23<00:00,  1.69it/s]
100%|██████████| 40/40 [00:21<00:00,  1.82it/s]
100%|██████████| 40/40 [00:22<00:00,  1.79it/s]


import matplotlib.pyplot as plt
import matplotlib.patches as patches
import pandas as pd
import os
from PIL import Image
from matplotlib.colors import ListedColormap
from matplotlib.colors import TABLEAU_COLORS


def plot_image_with_boxes_test(df, test_csv_path, id_column, bbox_columns, image_dir):
    """
    Plots images with bounding boxes for each unique ID in the dataframe, with all boxes for the same ID on one image.

    Args:
        df (pd.DataFrame): DataFrame containing bounding box information.
        test_csv_path (str): Path to the test.csv file.
        id_column (str): Column name in df corresponding to the ID.
        bbox_columns (list of str): Column names for bounding box coordinates ["xmin", "xmax", "ymin", "ymax"].
        image_dir (str): Path to the directory containing processed test images.
        scale (float): Scale factor for resizing the image (0 < scale <= 1).
    """
    # Load test.csv
    test_csv = pd.read_csv("/content/drive/MyDrive/processed_images/test.csv")

    scale = 0.2
    unique_ids = df[id_column].unique()
    num_images = len(unique_ids)

    # Create subplots
    fig, axes = plt.subplots(
        nrows=(num_images + 2) // 3, ncols=3, figsize=(15, 5 * ((num_images + 2) // 3))
    )
    axes = axes.flatten()

    # Create a color map for categories
    categories = df['category'].unique()
    category_colors = {category: plt.cm.get_cmap('tab10')(i % 10) for i, category in enumerate(categories)}

    for ax, img_id in zip(axes, unique_ids):
        img_data = df[df[id_column] == img_id]

        # Find the corresponding dcm name in test.csv
        dcm_name = test_csv.loc[test_csv['ID'] == img_id, "Filename"].values[0]
        dcm_name = os.path.splitext(dcm_name)[0]

        # Construct the image path
        image_path = os.path.join(image_dir, f"{dcm_name}.jpg")

        # Check if the image file exists
        if not os.path.exists(image_path):
            ax.set_title(f"Image {dcm_name} not found.")
            ax.axis('off')
            continue

        # Open the image
        image = Image.open(image_path)
        width, height = image.size

        # Resize the image based on scale
        image = image.resize((int(width * scale), int(height * scale)))
        width, height = image.size

        # Plot the image
        ax.imshow(image)

        # Draw bounding boxes
        for i, row in img_data.iterrows():
            xmin, xmax, ymin, ymax = [row[col] for col in bbox_columns]
            bounding_width = (xmax - xmin) * width
            bounding_height = (ymax - ymin) * height

            # Get the color for the category
            category = row.get("category", "Unknown")
            color = category_colors.get(category, 'gray')  # Default to gray if category is not in the map

            rect = patches.Rectangle(
                (xmin * width, ymin * height),
                bounding_width,
                bounding_height,
                linewidth=2,
                edgecolor=color,
                facecolor='none'
            )
            ax.add_patch(rect)

            # Display the category text near the bounding box
            ax.text(
                xmin * width,
                ymin * height - 10,
                category,
                color='white',
                fontsize=8,
                bbox=dict(facecolor=color, alpha=0.5)
            )

        ax.set_title(f"ID: {img_id}")
        ax.axis('off')

    # Turn off any unused subplots
    for ax in axes[num_images:]:
        ax.axis('off')

    plt.tight_layout()
    plt.show()


# Example usage
plot_image_with_boxes_test(merged_df[:10], "test.csv", "ID", ["xmin", "xmax", "ymin", "ymax"], "/content/drive/MyDrive/processed_images/test")

	ID	category	score	xmin	xmax	ymin	ymax
0	TDR02_20161123_145314	aortic_atherosclerosis_calcification	0.957472	0.521478	0.649914	0.202966	0.353814
1	TDR02_20161123_145314	lung_field_infiltration	0.876341	0.091065	0.983677	0.051271	0.853814
2	TDR02_20161123_145314	cardiac_hypertrophy	0.737568	0.369416	0.817440	0.377542	0.749576
3	TDR02_20161123_145314	aortic_curvature	0.707535	0.394759	0.661942	0.258475	0.685593
4	TDR01_20171106_111727	lung_field_infiltration	0.860005	0.045103	0.964347	0.055693	0.883663
5	TDR01_20171106_111727	degenerative_joint_disease_of_the_thoracic_spine	0.695168	0.354811	0.624141	0.030528	0.803630
6	TDR01_20171106_111727	scoliosis	0.664717	0.407646	0.603093	0.237211	0.879538
7	TDR01_20180510_090210	lung_field_infiltration	0.810126	0.095383	0.917247	0.117188	0.838816
8	TDR01_20180510_090210	degenerative_joint_disease_of_the_thoracic_spine	0.684162	0.374129	0.637631	0.056332	0.781250
9	TDR01_20180510_090210	scoliosis	0.678941	0.425087	0.595383	0.229030	0.886924

	ID	category	score	xmin	xmax	ymin	ymax
358	TDR02_20161125_122319	lung_field_infiltration	0.916460	0.110687	0.926527	0.057491	0.846690
359	TDR02_20161125_122319	intercostal_pleural_thickening	0.647039	0.220420	0.872137	0.087108	0.297909
360	TDR02_20180123_115426	degenerative_joint_disease_of_the_thoracic_spine	0.563914	0.368750	0.618750	0.061321	0.822720
361	TDR02_20180123_115426	lung_field_infiltration	0.852788	0.074167	0.942083	0.090802	0.885220
362	TDR01_20180508_173616	scoliosis	0.707762	0.406250	0.565755	0.244518	0.741776
363	TDR01_20180508_173616	lung_field_infiltration	0.611879	0.068359	0.953125	0.081689	0.805373
364	TDR02_20161118_145330	lung_field_infiltration	0.935987	0.090779	0.923004	0.050896	0.839577
365	TDR02_20161118_145330	cardiac_hypertrophy	0.771321	0.293251	0.789449	0.379886	0.687704
366	TDR02_20161118_145330	degenerative_joint_disease_of_the_thoracic_spine	0.674701	0.307034	0.605989	0.024837	0.767101
367	TDR02_20161118_145330	aortic_atherosclerosis_calcification	0.961850	0.434411	0.592205	0.241450	0.394544

	ID	category	score	xmin	xmax	ymin	ymax
0	TDR02_20161123_145314	lung_field_infiltration	0.813216	0.083333	0.962199	0.055508	0.850000
1	TDR02_20161123_145314	cardiac_hypertrophy	0.780251	0.401203	0.858677	0.340678	0.783051
2	TDR02_20161123_145314	aortic_curvature	0.759652	0.383162	0.678265	0.214407	0.711864
3	TDR02_20161123_145314	aortic_atherosclerosis_calcification	0.668996	0.513746	0.665808	0.189831	0.404661
4	TDR01_20171106_111727	lung_field_infiltration	0.801966	0.061426	0.947165	0.050743	0.839934
5	TDR01_20171106_111727	degenerative_joint_disease_of_the_thoracic_spine	0.656470	0.365120	0.623711	0.059818	0.781766
6	TDR01_20180510_090210	lung_field_infiltration	0.732217	0.089721	0.918554	0.087582	0.860197
7	TDR01_20180510_090210	scoliosis	0.719673	0.407230	0.602787	0.243010	0.858964
8	TDR01_20180511_092549	scoliosis	0.844056	0.413880	0.597826	0.263199	0.730202
9	TDR01_20180511_092549	lung_field_infiltration	0.727985	0.075669	0.942726	0.076087	0.839674

	ID	category	score	xmin	xmax	ymin	ymax
217	TDR02_20161125_122319	scoliosis	0.935547	0.418416	0.609256	0.225610	0.806620
218	TDR02_20161125_122319	lung_field_infiltration	0.706930	0.093989	0.928435	0.060976	0.851481
219	TDR02_20180123_115426	lung_field_infiltration	0.743149	0.064583	0.929583	0.064072	0.902909
220	TDR02_20180123_115426	degenerative_joint_disease_of_the_thoracic_spine	0.617154	0.371667	0.613750	0.021226	0.840016
221	TDR01_20180508_173616	scoliosis	0.862198	0.393880	0.597005	0.181469	0.763158
222	TDR01_20180508_173616	lung_field_infiltration	0.561979	0.079427	0.918620	0.043860	0.847588
223	TDR01_20180508_173616	degenerative_joint_disease_of_the_thoracic_spine	0.577343	0.347005	0.634115	0.026316	0.875000
224	TDR02_20161118_145330	lung_field_infiltration	0.795087	0.076996	0.856464	0.069218	0.855456
225	TDR02_20161118_145330	aortic_curvature	0.717589	0.303232	0.609316	0.245521	0.647394
226	TDR02_20161118_145330	cardiac_hypertrophy	0.871938	0.328422	0.764259	0.311075	0.715391

Object detection on medical image¶

Read data information¶

Categories¶

Images¶

Annotations¶

Data augmentation¶

Dataset¶

Collate_fn¶

Model: Faster R-CNN¶

Training¶

Validation¶

Main¶

SGD batch size=8¶

AdamW+Onecycle¶

TODO:¶

TODO2:¶